

import { listenDelayTask, publishToDelayTask } from '@nodejs/mq'
import { parseJSON } from '@tools/core'
import { ProxyCookies, ProxyUser, SpiderSubTask, SpiderTask, SpiderTaskStatus, useEntityManager } from '../entities/index'
import { EntityManager, In } from 'typeorm'
import dayjs, { Dayjs } from 'dayjs';
import { SpiderAuthor, SpiderAuthorType, SpiderLog, SpiderLogType, SpiderSubTaskStatus, SpiderTaskType } from '../entities/task';
import { v4 } from 'uuid';
import { createBrowserPage } from '../browser';
import { Browser, Page } from 'puppeteer';
import { load } from 'cheerio'
import axios, { AxiosHeaders } from 'axios';
import { delay } from '../util';
export async function spiderWeiBo(m: EntityManager, task: SpiderSubTask, clientId: string) {
    const { type, payload } = task;
    if (type === SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO || type === SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO_LIST) {
        const { url } = payload;
        await openSpiderKeywordSearchTopicWeiboUrl(clientId, url, m, task)
    }
    else if (type === SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO_DETAIL) {
        const { url } = payload
        const _ = new URL(url)
        const mid = _.pathname.split('/').pop()
        const targetUrl = `https://weibo.com/ajax/statuses/show?id=${mid}`;
        await openSpiderKeywordSearchTopicWeiboDetailUrl(clientId, targetUrl, m, task)
    }
    else if (type === SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO_LIKE) {
        const { uuid } = payload;
        await openSpiderKeywordSearchTopicWeiboLikeUrl(clientId, uuid, m, task)
    }
    else if (type === SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO_COMMENT) {
        const { uuid } = payload;
        await openSpiderKeywordSearchTopicWeiboCommentUrl(clientId, uuid, m, task)
    }
    else if (type === SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO_SHARE) {
        const { uuid } = payload;
        await openSpiderKeywordSearchTopicWeiboShareUrl(clientId, uuid, m, task)
    }
    else {
        await delay(1000 * 10)
        throw new Error(` no handler for type: ${type}`)
    }
}

async function openSpiderKeywordSearchTopicWeiboLikeUrl(clientId: string, uuid: string, m: EntityManager, task: SpiderSubTask) {
    const log = await m.findOne(SpiderLog, { where: { uuid } })
    if (!log) {
        return;
    }
    const id = log.log_id;
    const psize: number = 200;
    const headers = new AxiosHeaders()
    headers.setUserAgent(`Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36`)
    await useEntityManager(async m => {
        const user = await m.findOne(ProxyUser, { where: { login_at: clientId } })
        if (user) {
            const cookies = await m.find(ProxyCookies, { where: { user_id: user.uuid } })
            headers.set('cookie', cookies.map(c => `${c.name}=${c.value || ''}`).join('; '))
        }
    })
    async function getLikes(page: number = 1, likes: any[] = []) {
        const url = `https://weibo.com/ajax/statuses/likeShow?id=${id}&attitude_type=0&attitude_enable=1&page=${page}&count=${psize}`
        await axios.get(url, {
            headers: headers
        }).then(async res => {
            const { data, ok, total_number } = res.data;
            if (ok === -100) {
                // 今日休息
            }
            if (ok == 1 && data.length > 0) {
                likes.push(...data)
                const current = (page - 1) * psize
                if (total_number > current) {
                    await getLikes(page + 1, likes)
                }
            }
        }).catch(e => {
            console.error(`getLikes`, e)
        })
        return likes;
    }
    const likes = await getLikes()
    if (likes && likes.length > 0) {
        const users = likes.map(c => c.user);
        const authors = await m.find(SpiderAuthor, {
            where: {
                type: SpiderAuthorType.WEIBO,
                author_id: In(users.map(u => `${u.id}`))
            }
        });
        const toSaveAuthors = users.map(account => {
            const author = authors.find(it => it.author_id == `${account.id}`)
            return m.create(SpiderAuthor, {
                uuid: author?.uuid || v4(),
                author_id: `${account.id}`,
                type: SpiderAuthorType.WEIBO,
                detail: account
            })
        })
        if (toSaveAuthors && toSaveAuthors.length > 0) {
            await m.save(SpiderAuthor, toSaveAuthors)
        }
    }
    await m.update(SpiderLog, log.uuid, { detail: { ...log.detail, likes }, status: log.status + 1 })
    await m.update(SpiderSubTask, task.uuid, { status: SpiderSubTaskStatus.FINISH })
    if (task.parent_id) {
        const parentTask = await m.findOne(SpiderSubTask, { where: { uuid: task.parent_id } })
        if (parentTask) {
            await m.update(SpiderSubTask, task.parent_id, { status: parentTask.status + 1 })
        }
    }
}
/**
 * 评论
 */
async function openSpiderKeywordSearchTopicWeiboCommentUrl(clientId: string, uuid: string, m: EntityManager, task: SpiderSubTask) {
    const log = await m.findOne(SpiderLog, { where: { uuid } })
    if (!log) return;
    const author = await m.findOne(SpiderAuthor, { where: { uuid: log.author_id } })
    if (!author) return;
    const id = log.log_id;
    const uid = author.author_id;
    const psize: number = 200;
    const headers = new AxiosHeaders()
    headers.setUserAgent(`Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36`)
    await useEntityManager(async m => {
        const user = await m.findOne(ProxyUser, { where: { login_at: clientId } })
        if (user) {
            const cookies = await m.find(ProxyCookies, { where: { user_id: user.uuid } })
            headers.set('cookie', cookies.map(c => `${c.name}=${c.value || ''}`).join('; '))
        }
    })
    function createUrl(url: string, query: any) {
        const str = Object.keys(query).map(key => {
            return `${key}=${query[key]}`
        }).join('&')
        return `${url}?${str}`
    }
    async function getComments(id: string, uid: string, max_id: number, comments: any[] = []) {
        const url = `https://weibo.com/ajax/statuses/buildComments`
        const query: any = {};
        query.is_reload = 1;
        query.id = id;
        query.count = psize;
        query.uid = uid;
        query.is_show_bulletin = 2;
        if (max_id) {
            query.flow = 0;
            query.is_mix = 0;
            query.max_id = max_id
        } else {
            query.is_mix = 0;
        }
        await axios.get(createUrl(url, query), {
            headers: headers
        }).then(async res => {
            const { data, ok, max_id } = res.data;
            if (ok === 1 && data.length > 0) {
                const list = await Promise.all(data.map(async (item: any) => {
                    const children = await getChildrenComments(item.id, uid, 0).catch(e => {
                        console.error(`getChildrenComments error`, e)
                        return [];
                    })
                    let users: any[] = [];
                    item.children = children.map(child => {
                        const { user } = child;
                        users.push(user)
                        return child;
                    })
                    if (users.length > 0) {
                        const authors = await m.find(SpiderAuthor, {
                            where: {
                                type: SpiderAuthorType.WEIBO,
                                author_id: In(users.map(u => `${u.id}`))
                            }
                        });
                        const toSaveAuthors = users.map(account => {
                            const author = authors.find(it => it.author_id == `${account.id}`)
                            return m.create(SpiderAuthor, {
                                uuid: author?.uuid || v4(),
                                author_id: `${account.id}`,
                                type: SpiderAuthorType.WEIBO,
                                detail: account
                            })
                        })
                        if (toSaveAuthors && toSaveAuthors.length > 0) {
                            await m.save(SpiderAuthor, toSaveAuthors)
                        }
                    }
                    return item;
                }))
                comments.push(...list);
                if (max_id) {
                    await getComments(id, uid, max_id, comments).catch(e => console.error(`getComments error`, e))
                }
            }
        }).catch(e => console.error(`getComments error`, e))
        if (comments && comments.length > 0) {
            const users = comments.map(c => c.user);
            const authors = await m.find(SpiderAuthor, {
                where: {
                    type: SpiderAuthorType.WEIBO,
                    author_id: In(users.map(u => `${u.id}`))
                }
            });
            const toSaveAuthors = users.map(account => {
                const author = authors.find(it => it.author_id == `${account.id}`)
                return m.create(SpiderAuthor, {
                    uuid: author?.uuid || v4(),
                    author_id: `${account.id}`,
                    type: SpiderAuthorType.WEIBO,
                    detail: account
                })
            })
            if (toSaveAuthors && toSaveAuthors.length > 0) {
                await m.save(SpiderAuthor, toSaveAuthors)
            }
        }
        return comments;
    }
    async function getChildrenComments(id: string, uid: string, max_id: number, comments: any[] = []) {
        const url = `https://weibo.com/ajax/statuses/buildComments`
        const query: any = {};
        query.is_reload = 1;
        query.id = id
        query.is_show_bulletin = 2;
        query.is_mix = 1;
        query.fetch_level = 1;
        query.count = 100;
        query.uid = uid;
        if (max_id) {
            query.flow = 0;
            query.max_id = max_id;
        } else {
            query.max_id = 0
        }
        await axios.get(createUrl(url, query), {
            headers: headers
        }).then(async res => {
            const { data, ok, max_id } = res.data;
            if (ok === 1 && data.length > 0) {
                comments.push(...data);
                if (max_id) {
                    await getChildrenComments(id, uid, max_id, comments)
                }
            }
        })
        return comments;
    }
    const comments = await getComments(id, uid, 0)
    await m.update(SpiderLog, log.uuid, { detail: { ...log.detail, comments }, status: log.status + 1 })
    await m.update(SpiderSubTask, task.uuid, { status: SpiderSubTaskStatus.FINISH })
    if (task.parent_id) {
        const parentTask = await m.findOne(SpiderSubTask, { where: { uuid: task.parent_id } })
        if (parentTask) {
            await m.update(SpiderSubTask, task.parent_id, { status: parentTask.status + 1 })
        }
    }
}

async function openSpiderKeywordSearchTopicWeiboShareUrl(clientId: string, uuid: string, m: EntityManager, task: SpiderSubTask) {
    const log = await m.findOne(SpiderLog, { where: { uuid } })
    if (!log) return;
    const author = await m.findOne(SpiderAuthor, { where: { uuid: log.author_id } })
    if (!author) return;
    const id = log.log_id;
    const uid = author.author_id;
    const psize: number = 200;
    const headers = new AxiosHeaders()
    headers.setUserAgent(`Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36`)
    await useEntityManager(async m => {
        const user = await m.findOne(ProxyUser, { where: { login_at: clientId } })
        if (user) {
            const cookies = await m.find(ProxyCookies, { where: { user_id: user.uuid } })
            headers.set('cookie', cookies.map(c => `${c.name}=${c.value || ''}`).join('; '))
        }
    })
    async function getRepostTimeLine(page: number = 1, likes: any[] = []) {
        const url = `https://weibo.com/ajax/statuses/repostTimeline?id=${id}&page=${page}&moduleID=feed&count=${psize}`
        await axios.get(url, {
            headers: headers
        }).then(async res => {
            const { data, ok, total_number } = res.data;
            if (ok === -100) {
                // 今日休息
            }
            if (ok == 1 && data.length > 0) {
                likes.push(...data)
                const current = (page - 1) * psize
                if (total_number > current) {
                    await getRepostTimeLine(page + 1, likes)
                }
            }
        }).catch(e => {
            console.error(`getRepostTimeLine error`, e)
        })
        return likes;
    }
    const shares = await getRepostTimeLine()
    if (shares && shares.length > 0) {
        const users = shares.map(c => c.user);
        const authors = await m.find(SpiderAuthor, {
            where: {
                type: SpiderAuthorType.WEIBO,
                author_id: In(users.map(u => `${u.id}`))
            }
        });
        const toSaveAuthors = users.map(account => {
            const author = authors.find(it => it.author_id == `${account.id}`)
            return m.create(SpiderAuthor, {
                uuid: author?.uuid || v4(),
                author_id: `${account.id}`,
                type: SpiderAuthorType.WEIBO,
                detail: account
            })
        })
        if (toSaveAuthors && toSaveAuthors.length > 0) {
            await m.save(SpiderAuthor, toSaveAuthors)
        }
    }
    await m.update(SpiderLog, log.uuid, { detail: { ...log.detail, shares }, status: log.status + 1 })
    await m.update(SpiderSubTask, task.uuid, { status: SpiderSubTaskStatus.FINISH })
    if (task.parent_id) {
        const parentTask = await m.findOne(SpiderSubTask, { where: { uuid: task.parent_id } })
        if (parentTask) {
            await m.update(SpiderSubTask, task.parent_id, { status: parentTask.status + 1 })
        }
    }
}

/**
 * 详情
 */
async function openSpiderKeywordSearchTopicWeiboDetailUrl(clientId: string, url: string, m: EntityManager, task: SpiderSubTask) {
    const headers = new AxiosHeaders()
    await useEntityManager(async m => {
        const user = await m.findOne(ProxyUser, { where: { login_at: clientId } })
        if (user) {
            const cookies = await m.find(ProxyCookies, { where: { user_id: user.uuid } })
            headers.set('cookie', cookies.map(c => `${c.name}=${c.value || ''}`).join('; '))
        }
    })
    headers.setUserAgent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36')
    await axios.get(url, {
        headers: headers
    }).then(async res => {
        const { user: account, ok, error_code, message, ...content } = res.data;
        if (ok == 1) {
            const author = await m.findOne(SpiderAuthor, {
                where: {
                    type: SpiderAuthorType.WEIBO,
                    author_id: `${account.id}`
                }
            })
            const toSaveAuthor = m.create(SpiderAuthor, {
                uuid: author?.uuid || v4(),
                author_id: `${account.id}`,
                type: SpiderAuthorType.WEIBO,
                detail: account
            })
            await m.save(SpiderAuthor, toSaveAuthor)
            const one = await m.findOne(SpiderLog, {
                where: {
                    type: SpiderLogType.WEIBO_TOPIC,
                    log_id: `${content.id}`,
                }
            })
            const log = m.create(SpiderLog, {
                uuid: one?.uuid || v4(),
                type: SpiderLogType.WEIBO_TOPIC,
                log_id: `${content.id}`,
                detail: content,
                author_id: toSaveAuthor.uuid,
                task_id: task.uuid,
                status: 1
            });
            await m.save(SpiderLog, log)
            // 创建 like comment share
            await m.save(SpiderSubTask, m.create(SpiderSubTask, {
                payload: { uuid: log.uuid },
                type: SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO_LIKE,
                uuid: v4(),
                task_id: task.task_id,
                status: SpiderSubTaskStatus.UNFINISH,
                displayorder: task.displayorder,
                parent_id: task.uuid,
            }))
            await m.save(SpiderSubTask, m.create(SpiderSubTask, {
                payload: { uuid: log.uuid },
                type: SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO_COMMENT,
                uuid: v4(),
                task_id: task.task_id,
                status: SpiderSubTaskStatus.UNFINISH,
                displayorder: task.displayorder,
                parent_id: task.uuid,
            }))
            await m.save(SpiderSubTask, m.create(SpiderSubTask, {
                payload: { uuid: log.uuid },
                type: SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO_SHARE,
                uuid: v4(),
                task_id: task.task_id,
                status: SpiderSubTaskStatus.UNFINISH,
                displayorder: task.displayorder,
                parent_id: task.uuid,
            }))
        }
    }).catch(e => {
        throw e;
    })
}

/**
 * 获取分页链接和详情链接
 */
async function openSpiderKeywordSearchTopicWeiboUrl(clientId: string, url: string, m: EntityManager, task: SpiderSubTask) {
    let page: Page | null = null;
    let browser: Browser | null = null;
    function parseUrl(url: string) {
        if (!url) {
            return;
        }
        if (url.startsWith('https')) {
            return url
        } else {
            return `https:${url}`
        }
    }
    try {
        const [_page, _browser] = await createBrowserPage(clientId, url)
        page = _page;
        browser = _browser;
        // 获取当前页码和总页码
        const content = await page.content()
        const $ = load(content);
        const links = $('#pl_feedlist_index .card-feed')
        const detailUrls: string[] = []
        links.map((index, item) => {
            const $ = load(item);
            const linkElement = $('.content > .from > a');
            const link = linkElement[0]?.attribs['href'];
            const url = parseUrl(link)
            if (url) {
                detailUrls.push(url)
            }
        });
        if (detailUrls && detailUrls.length > 0) {
            const subTasks = detailUrls.map(url => {
                return m.create(SpiderSubTask, {
                    payload: { url },
                    type: SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO_DETAIL,
                    uuid: v4(),
                    task_id: task.task_id,
                    status: SpiderSubTaskStatus.UNFINISH,
                    displayorder: task.displayorder
                })
            })
            await m.save(SpiderSubTask, subTasks)
        }

        if (task.type === SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO) {
            const pageUrls: string[] = [];
            const moreList = $('ul[node-type="feed_list_page_morelist"] > li')
            moreList.map((index, el) => {
                const cur = el.attribs['class'] && el.attribs['class'].includes('cur')
                if (!cur) {
                    const $ = load(el)
                    const a = $('a')[0]
                    const href = a.attribs['href'];
                    const url = parseUrl(`https://s.weibo.com${href}`)!
                    if (url) pageUrls.push(url)
                }
            });
            if (pageUrls && pageUrls.length > 0) {
                const subTasks = pageUrls.map(url => {
                    return m.create(SpiderSubTask, {
                        payload: { url },
                        type: SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO_LIST,
                        uuid: v4(),
                        task_id: task.task_id,
                        status: SpiderSubTaskStatus.UNFINISH,
                        displayorder: task.displayorder,
                    })
                })
                await m.save(SpiderSubTask, subTasks)
            }
        }
        await m.update(SpiderSubTask, task.uuid, { status: SpiderSubTaskStatus.FINISH })
    } catch (e) {
        throw e;
    } finally {
        if (page) {
            await page.close()
        }
    }
}

function createSpiderKeywordSearchTopicWeiboUrl(keyword: string, startDate: Dayjs, endDate: Dayjs) {
    const timescope = `custome:${startDate.format('YYYY-MM-DD')}-${startDate.hour()}:${endDate.format('YYYY-MM-DD')}-${endDate.hour()}`
    let url = `https://s.weibo.com/weibo?q=${encodeURIComponent(keyword)}&timescope=${timescope}&Refer=g&typeall=1&`
    return url;
}

export async function listenSpiderWeiBo(clientId: string) {
    // SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO
    return await listenDelayTask('spider.weibo', async (msg, c) => {
        try {
            const content = msg.content.toString('utf8')
            const data = parseJSON(content)
            const { taskId } = data;
            await useEntityManager(async m => {
                const task = await m.findOne(SpiderTask, { where: { uuid: taskId } })
                if (task) {
                    const { step, keyword } = task.payload;
                    const subTask = await m.findOne(SpiderSubTask, {
                        where: {
                            task_id: task.uuid, status: SpiderSubTaskStatus.UNFINISH
                        }, order: { displayorder: 'desc' }
                    })
                    if (subTask) {
                        // 执行搜索任务
                        await spiderWeiBo(m, subTask, clientId)
                        console.log(`sub task ${subTask.uuid} finish`)
                        await delay(1000 * 3)
                        c.nack(msg)
                    } else {
                        await m.update(SpiderTask, task.uuid, { status: SpiderTaskStatus.STOP })
                        c.ack(msg)
                    }
                    // 发送消息
                    const start = dayjs(task.sub_to_date).startOf('hour')
                    const end = dayjs(task.sub_to_date).add(step, 'hour').startOf('hour')
                    if (end.toDate() < dayjs().toDate()) {
                        const url = createSpiderKeywordSearchTopicWeiboUrl(keyword, start, end)
                        const uuid = v4();
                        const entity = m.create(SpiderSubTask, {
                            uuid: uuid,
                            displayorder: Date.now(),
                            task_id: task.uuid,
                            payload: {
                                url
                            },
                            status: SpiderSubTaskStatus.UNFINISH,
                            type: SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO,
                        });
                        await m.save(SpiderSubTask, entity)
                        await m.update(SpiderTask, task.uuid, { sub_to_date: end.toDate() })
                        const buf = Buffer.from(JSON.stringify({
                            taskId: task.uuid,
                        }));
                        const now = dayjs();
                        const expires = Math.floor((end.toDate().getTime() - now.toDate().getTime()) / 1000)
                        await publishToDelayTask('spider.weibo', buf, expires)
                    }
                } else {
                    console.log(`任务不存在`)
                    c.ack(msg)
                }
            })
        } catch (e) {
            console.error(`spider.weibo error`, e)
            c.nack(msg)
        }
    })
}

/**
 * 创建任务
 */
export async function createSpiderKeywordSearchTopicWeiboTask(keyword: string, start: string, step: number) {
    await useEntityManager(async m => {
        const endDate = dayjs().startOf('hour')
        let startDate = dayjs(start)
        if (endDate < startDate) {
            throw new Error(`开始时间不能大于当前时间`)
        }
        let stepEndDate = startDate.add(step, 'hour')
        let urls: { url: string, start: Dayjs, end: Dayjs }[] = [];
        while (stepEndDate < endDate) {
            urls.push({
                url: createSpiderKeywordSearchTopicWeiboUrl(keyword, startDate, stepEndDate),
                start: startDate,
                end: endDate
            })
            startDate = stepEndDate;
            stepEndDate = startDate.add(step, 'hour')
        }
        const uuid = v4()
        const task = m.create(SpiderTask, {
            uuid: uuid,
            status: SpiderTaskStatus.WACHING,
            type: SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO,
            displayorder: Date.now(),
            sub_to_date: stepEndDate.toDate(),
            payload: {
                keyword: keyword,
                start: start,
                step
            }
        })
        await m.save(SpiderTask, task)
        const subTasks = urls.map(url => {
            return m.create(SpiderSubTask, {
                uuid: v4(),
                payload: { url: url.url },
                task_id: task.uuid,
                displayorder: url.end.toDate().getTime(),
                status: SpiderSubTaskStatus.UNFINISH,
                type: SpiderTaskType.SPIDER_KEYWORD_SEARCH_TOPIC_WEIBO,
            })
        })
        await m.save(SpiderSubTask, subTasks)
        const buf = Buffer.from(JSON.stringify({
            taskId: task.uuid,
        }));
        await publishToDelayTask('spider.weibo', buf, 10 * 1000)
    })
}